{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SOSNet: Second Order Similarity Regularization for Local Descriptor Learning\n",
    "\n",
    "## Runing `SOSNet` with `openCV` for image matching\n",
    "\n",
    "Below we show how to use the `openCV` pipeline to match two images using `SOSNet`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "import sosnet_model\n",
    "import os\n",
    "%matplotlib notebook\n",
    "from matplotlib import pyplot as plt\n",
    "import cv2\n",
    "import tfeat_utils\n",
    "import numpy as np\n",
    "import cv2\n",
    "torch.no_grad()\n",
    "\n",
    "# Init the 32x32 version of SOSNet \n",
    "sosnet32 = sosnet_model.SOSNet32x32()\n",
    "net_name = 'notredame'\n",
    "sosnet32.load_state_dict(torch.load(os.path.join('sosnet-weights',\"sosnet-32x32-\"+net_name+\".pth\")))\n",
    "sosnet32.cuda().eval();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<img src=\"\" width=\"640\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "(<Figure size 640x480 with 1 Axes>,\n",
       " <matplotlib.image.AxesImage at 0x7f981c0642e8>,\n",
       " None)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the images and detect BRISK keypoints using openCV\n",
    "img1 = cv2.imread('imgs/img1.jpg', 0)\n",
    "img2 = cv2.imread('imgs/img2.jpg', 0)\n",
    "\n",
    "brisk = cv2.BRISK_create(100)\n",
    "kp1 = brisk.detect(img1, None)\n",
    "kp2 = brisk.detect(img2, None)\n",
    "\n",
    "# We use the tfeat_utils methods that rectify patches around openCV keypoints and \n",
    "desc_tfeat1 = tfeat_utils.describe_opencv(sosnet32, img1, kp1, patch_size=32, mag_factor=3)\n",
    "desc_tfeat2 = tfeat_utils.describe_opencv(sosnet32, img2, kp2, patch_size=32, mag_factor=3)\n",
    "\n",
    "bf = cv2.BFMatcher(cv2.NORM_L2)\n",
    "matches = bf.knnMatch(desc_tfeat1,desc_tfeat2, k=2)\n",
    "\n",
    "# Apply SIFT's ratio test, notice that 0.8 may not be the best ratio for SOSNet\n",
    "good = []\n",
    "for m,n in matches:\n",
    "    if m.distance < 0.8*n.distance:\n",
    "        good.append([m])\n",
    "        \n",
    "img_matches_32 = cv2.drawMatchesKnn(img1, kp1, img2, kp2, good, 0, flags=2)\n",
    "cv2.imwrite(\"sosnet32-matches.png\", img_matches_32)\n",
    "plt.figure(0), plt.imshow(img_matches_32), plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
